/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

/*
 * RLLGui.java
 *
 * Created on 2010-07-10, 15:16:16
 */
package gui;

import java.lang.reflect.Constructor;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartPanel;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.data.xy.XYSeries;
import org.jfree.data.xy.XYSeriesCollection;
import rll.QLearning;
import rll.Sarsa;
import testenv.Action;
import testenv.Environment;
import testenv.TestAction;
import testenv.WorldState;

/**
 *
 * @author piotrrr
 */
public class RLLGui extends javax.swing.JFrame {

    /** Creates new form RLLGui */
    public RLLGui() {
        initComponents();
    }

    /** This method is called from within the constructor to
     * initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is
     * always regenerated by the Form Editor.
     */
    @SuppressWarnings("unchecked")
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        stepsTextField = new javax.swing.JTextField();
        runjButton1 = new javax.swing.JButton();
        jTabbedPane1 = new javax.swing.JTabbedPane();
        chart1 = new javax.swing.JPanel();
        resetNRunjButton2 = new javax.swing.JButton();
        betaTextField1 = new javax.swing.JTextField();
        gammajTextField2 = new javax.swing.JTextField();
        explorationTextField3 = new javax.swing.JTextField();
        jLabel1 = new javax.swing.JLabel();
        jLabel2 = new javax.swing.JLabel();
        jLabel3 = new javax.swing.JLabel();
        applyButton = new javax.swing.JButton();
        experimentsTextField1 = new javax.swing.JTextField();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);

        stepsTextField.setText("2000");
        stepsTextField.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                stepsTextFieldActionPerformed(evt);
            }
        });

        runjButton1.setText("run");
        runjButton1.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                runjButton1ActionPerformed(evt);
            }
        });

        chart1.setLayout(new java.awt.GridLayout(1, 0));
        jTabbedPane1.addTab("general", chart1);

        resetNRunjButton2.setText("reset RLBot & run");
        resetNRunjButton2.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                resetNRunjButton2ActionPerformed(evt);
            }
        });

        betaTextField1.setText("0.2");

        gammajTextField2.setText("0.9");

        explorationTextField3.setText("0.1");

        jLabel1.setText("gamma");

        jLabel2.setText("beta");

        jLabel3.setText("exploration");

        applyButton.setText("apply");
        applyButton.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                applyButtonActionPerformed(evt);
            }
        });

        experimentsTextField1.setText("25");

        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(jTabbedPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 500, Short.MAX_VALUE)
                    .addGroup(layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING, false)
                            .addComponent(stepsTextField, javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(runjButton1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(resetNRunjButton2)
                            .addComponent(experimentsTextField1, javax.swing.GroupLayout.PREFERRED_SIZE, 50, javax.swing.GroupLayout.PREFERRED_SIZE))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
                            .addComponent(betaTextField1, javax.swing.GroupLayout.Alignment.TRAILING)
                            .addComponent(gammajTextField2, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, 28, Short.MAX_VALUE))
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addGroup(layout.createSequentialGroup()
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                .addComponent(jLabel1))
                            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                .addComponent(jLabel2)))
                        .addGap(18, 18, 18)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addGroup(layout.createSequentialGroup()
                                .addComponent(explorationTextField3, javax.swing.GroupLayout.PREFERRED_SIZE, 28, javax.swing.GroupLayout.PREFERRED_SIZE)
                                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                .addComponent(jLabel3))
                            .addComponent(applyButton))))
                .addContainerGap())
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addGroup(layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(stepsTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(gammajTextField2, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(jLabel1)
                            .addComponent(experimentsTextField1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(runjButton1)
                            .addComponent(resetNRunjButton2)
                            .addComponent(betaTextField1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(jLabel2)))
                    .addGroup(layout.createSequentialGroup()
                        .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(explorationTextField3, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(jLabel3))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(applyButton)))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(jTabbedPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 312, Short.MAX_VALUE)
                .addContainerGap())
        );

        pack();
    }// </editor-fold>//GEN-END:initComponents

    private void runjButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_runjButton1ActionPerformed

        int steps = Integer.parseInt(stepsTextField.getText());
        int experiments = Integer.parseInt(experimentsTextField1.getText());

        resetRewardInTimeChart();

//        doTheExperiment(rcRand, new RandomBot(), experiments, steps);

        doTheExperiment(rcRef, new ReferenceBot(), experiments, steps);

//        doTheExperiment(rcRL, new QLearningBot(), experiments, steps);

        doTheExperiment(rcCRL, new ConnQLBot(), experiments, steps);

//        System.out.println(rlbot.qf.toString());

    }//GEN-LAST:event_runjButton1ActionPerformed

    private void stepsTextFieldActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_stepsTextFieldActionPerformed
        // TODO add your handling code here:
    }//GEN-LAST:event_stepsTextFieldActionPerformed

    private void resetNRunjButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_resetNRunjButton2ActionPerformed
        applyButtonActionPerformed(evt);
        runjButton1ActionPerformed(evt);
    }//GEN-LAST:event_resetNRunjButton2ActionPerformed

    private void applyButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_applyButtonActionPerformed
        QLearning.setParameters(Double.parseDouble(gammajTextField2.getText()),
                Double.parseDouble(betaTextField1.getText()),
                Double.parseDouble(explorationTextField3.getText()));
        Sarsa.setParameters(Double.parseDouble(gammajTextField2.getText()),
                Double.parseDouble(betaTextField1.getText()),
                Double.parseDouble(explorationTextField3.getText()));
    }//GEN-LAST:event_applyButtonActionPerformed

    /**
     * @param args the command line arguments
     */
    public static void main(String args[]) {
        java.awt.EventQueue.invokeLater(new Runnable() {

            public void run() {
                RLLGui gui = new RLLGui();
                gui.setVisible(true);
                gui.resetNRunjButton2ActionPerformed(null);
            }
        });
    }
    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton applyButton;
    private javax.swing.JTextField betaTextField1;
    private javax.swing.JPanel chart1;
    private javax.swing.JTextField experimentsTextField1;
    private javax.swing.JTextField explorationTextField3;
    private javax.swing.JTextField gammajTextField2;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JLabel jLabel2;
    private javax.swing.JLabel jLabel3;
    private javax.swing.JTabbedPane jTabbedPane1;
    private javax.swing.JButton resetNRunjButton2;
    private javax.swing.JButton runjButton1;
    private javax.swing.JTextField stepsTextField;
    // End of variables declaration//GEN-END:variables
    XYSeries rcRand;
    XYSeries rcRef;
    XYSeries rcRL;
    XYSeries rcCRL;
    int parts = 500;

    private void doTheExperiment(XYSeries series, Bot bot, int expCount, int steps) {

        dbg("Starting for bot " + bot.getClass().getName());

        double[][] rewards = new double[steps][expCount];

        double lastpercent = 0;

        for (int expNum = 0; expNum < expCount; expNum++) {

            double percent = (double) expNum / (double) expCount;
            if (percent > lastpercent) {
                lastpercent += 0.1;
                dbg("Finished " + percent * 100 + "%");
            }

            for (Constructor c : bot.getClass().getConstructors()) {
                if (c.getParameterTypes().length == 0) {
                    try {
                        bot = (Bot) c.newInstance();
                    } catch (Exception ex) {
                        System.err.println("Cannot reset the bot class!");
                    }
                }
            }

            applyButtonActionPerformed(null);

            Environment.resetWorld();
            double reward = 0;
            WorldState state = Environment.getNextState(new TestAction(Action.NO_ACTION));
            for (int step = 0; step < steps; step++) {
                state = Environment.getNextState(bot.getAction(state));
                if (state.getLastReward() != 0) {
                    reward += state.getLastReward();
                }
                rewards[step][expNum] = reward;
            }
        }

        int partsize = steps/parts;
        if (partsize <= 0) partsize = 1;

        double sum = 0;
        for (int step = 0; step < steps; step++) {
            sum += StatMethods.getMean(rewards[step]);
            if (step % partsize == 0) {
                series.add(step, sum/partsize);
                sum = 0;
            }
        }

        if (bot.getClass().equals(QLearningBot.class)) {
            dbg(((QLearningBot) bot).learner.toString());
        }
    }

    public void resetRewardInTimeChart() {
        Environment.resetWorld();
        XYSeriesCollection ds = new XYSeriesCollection();

        rcRand = new XYSeries("random");
        ds.addSeries(rcRand);

        rcRef = new XYSeries("reference");
        ds.addSeries(rcRef);

        rcRL = new XYSeries("QLearning");
        ds.addSeries(rcRL);

        rcCRL = new XYSeries("ConnectionistQL");
        ds.addSeries(rcCRL);

        JFreeChart c = ChartFactory.createXYLineChart(
                "Total reward in time",
                "time [1/10 s]",
                "total reward",
                ds,
                PlotOrientation.VERTICAL,
                true, true, true);

        ChartPanel cp = new ChartPanel(c);
        chart1.removeAll();
        chart1.add(cp);
        chart1.validate();

    }

    private void dbg(Object o) {
        System.out.println(o.toString());
        System.out.flush();
    }
}
